package com.xiaotu.spider.project.pageprocessor.nettv;

import com.xiaotu.common.exception.SpiderException;
import com.xiaotu.common.exception.SpiderExceptionCode;
import com.xiaotu.common.model.NetTVModel;
import com.xiaotu.common.model.TVCityModel;
import com.xiaotu.common.util.*;
import com.xiaotu.spider.AUrlPageProcessor;
import com.xiaotu.spider.iqy.pageprocessor.PeoplePageProcessor;

import net.sf.json.JSONArray;
import net.sf.json.JSONObject;
import org.apache.commons.lang.StringUtils;
import org.springframework.stereotype.Component;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.selector.Html;

import java.math.BigDecimal;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Stream;
/**
 * 
 * 项目名称：Sprider-WebMagic    
 * 类名称：IQYNetTVPageProcessor    
 * 类描述： 爱奇艺最新三年网剧爬取：名称、集数、类型、导演、主演、语言、年份、简介、播放量、评分   
 * 创建人：张晓   
 * 创建时间：2020年1月6日 下午3:51:31    
 * @version
 */
@Component("IQYNetTVPageProcessor")
public class IQYNetTVPageProcessor extends AUrlPageProcessor
{
    
	//“所有电视剧”列表的页面地址，"%s"为要替换的年份
    private static final String START_URL =
            "http://list.iqiyi.com";
    
    private static final String BASE_URL =
            "http://pcw-api.iqiyi.com/search/video/videolists?"
            + "access_play_control_platform=14&channel_id=2&data_type=1"
            + "&from=pcw_list&is_album_finished=&is_purchase=&"
            + "key=&market_release_date_level=%s&mode=11"
            + "&pageNum=%s&pageSize=48"
            + "&site=iqiyi&source_type=&three_category_id=24065;must,%s;must&without_qipu=1";
    //获取地区，语言
    private static final String AEREA_URL =
    		"http://pcw-api.iqiyi.com/video/video/playervideoinfo?"
    		+ "tvid=%s&locale=cn_s&"
    		+ "callback=Q5b07eeea4adcefae87245d589d905dc0";
    //获取导演
    private static final String DIRECTOR_URL =
    		"http://pcw-api.iqiyi.com/album/album/trailerlist?"
    		+ "aid=%s&size=1&"
    		+ "callback=jsonp_1578039316166_66628";
    

    //数据范围，只需要爬取这三年的网剧，该变量将被逐个拼到url中
    private static final String[] YEARS = DateUtils.getRecentYears(3);
    
    //地区
    private static final String[][] AREAS = {{"15","16","1117"},{"内地","香港","台湾"}};//内地,香港,台湾
    

    /**
     * 返回本页面的数据类型
     *
     * @param page 爬取页面
     * @return
     */
    @Override
    public String getPageDataType(Page page)
    {
        String url = page.getUrl().toString();
        if (url.contains("list.iqiyi.com"))//网剧列表页面
            return Constants.PageDataType.SEARCH_LIST;
        if (url.contains("get_user_profile"))
            return Constants.PageDataType.TV_AUDIENCE;
        if (url.contains("get_province_distribution"))
            return Constants.PageDataType.TV_CITY;
        if (url.contains("http://pcw-api.iqiyi.com/search/video/videolists?"))
            return Constants.PageDataType.TV_BASICINFO;
        if (url.contains("http://pcw-api.iqiyi.com/video/video/playervideoinfo?tvid="))//获取地区，语言
            return "playervideoinfo";
        if (url.contains("http://pcw-api.iqiyi.com/album/album/trailerlist?"))//获取导演
        	return "album";
        if (url.contains("http://www.iqiyi.com/v_"))//网剧详情页面
        	return Constants.PageDataType.TV_RECORD;
        return StringUtils.EMPTY;
    }
    
    /**
     * 页面内容被爬虫框架捕获后，将回调此方法，来完成页面数据的解析
     *
     * @param page
     */
    @Override
    public void process(Page page)
    {
    	
    	Map<String,Object> extraMap = page.getRequest().getExtras();
    	
        String type = this.getPageDataType(page);
        switch (type)
        {
            case Constants.PageDataType.SEARCH_LIST:
                this.listProcess(page);
                break;
            case Constants.PageDataType.TV_BASICINFO:
            	this.basicinfoProcess(page,extraMap);
            	break;
            case Constants.PageDataType.TV_AUDIENCE:
                this.audienceProcess(page);
                break;
            case Constants.PageDataType.TV_CITY:
                this.cityProcess(page);
                break;
            case "playervideoinfo":
            	this.areaAndLangProcess(page,extraMap);
            	break;
            case "album":
            	this.directorProcess(page,extraMap);
            	break;
            case Constants.PageDataType.TV_RECORD:
                this.detailProcess(page,extraMap);
                break;
        }
    }
    
    /**
     * 将列表中的所有网剧添加到爬取池中
     *
     * @param page
     */
    @SuppressWarnings("unchecked")
	private void listProcess(Page page)
    {
    	
    	for(String year : YEARS){
    		for(int j=0;j<AREAS[0].length;j++){
    			String areaValue = AREAS[0][j];
    			String areaName = AREAS[1][j];
    			String json = HttpUtils.httpGetJSONP(String.format(BASE_URL, year, 1 , areaValue));
        		if (StringUtils.isEmpty(json))
                    throw new SpiderException(SpiderExceptionCode.DataInvalideException);
        		Map<String,Object> jsonMap = GsonUtils.fromJson(json, Map.class);
        		Map<String,Object> dataMap = (Map<String,Object>)jsonMap.get("data");
        		if (null == dataMap)
                    throw new SpiderException(SpiderExceptionCode.DataInvalideException);
        		Integer pageTotal = Integer.parseInt((dataMap.get("pageTotal")+StringUtils.EMPTY).replace(".0", ""));
        		
        		if(pageTotal>=1){
        			for(int i=1;i<=pageTotal;i++){
        				Map<String,Object> extraMap = new HashMap<String,Object>();
        				extraMap.put("year", year);
        				extraMap.put("area", areaName);
        				PageUtils.addTargetRequest(page, String.format(BASE_URL, year, i , areaValue),extraMap);
            		}
        		}
    		}
    	}
    	
    }

    /** directorProcess(获取导演)   
	 * 创建人：张晓   
	 * 创建时间：2020年1月3日 下午4:54:02    
	 * 修改备注： 
	 * @param page
	 * @param extraMap  
	 */
	private void directorProcess(Page page, Map<String, Object> extraMap) {
		NetTVModel netTVModel = (NetTVModel)extraMap.get(NetTVModel.class.getName());
		String json = page.getRawText();
		String director = StringUtils.EMPTY;
		try {
	        	if(StringUtils.isNotBlank(json)){
	        		json = json.substring(json.indexOf("(")+1,json.lastIndexOf(");"));
	        		JSONObject obj = GsonUtils.fromJson(json, JSONObject.class);
	        		JSONArray directorArray = obj.getJSONArray("data").getJSONObject(0).getJSONObject("people").getJSONArray("director");
	        		for(int i=0;i<directorArray.size();i++){
	        			JSONObject o = directorArray.optJSONObject(i);
	        			director += ","+o.getString("name");
	        		}
	            }
			} catch (Exception e) {
		}
		if(null != director && StringUtils.isNotBlank(director)){
			director = director.substring(1);
		}
		netTVModel.setDirector(director);
		page.putField(NetTVModel.class.getName(), netTVModel);
	}

	/** areaAndLangProcess(获取地区，语言)   
	 * 创建人：张晓   
	 * 创建时间：2020年1月3日 下午4:54:00    
	 * 修改备注： 
	 * @param page
	 * @param extraMap  
	 */
	private void areaAndLangProcess(Page page, Map<String, Object> extraMap) {
		NetTVModel netTVModel = (NetTVModel)extraMap.get(NetTVModel.class.getName());
		//网站tvid
        String tvId = extraMap.get("tvId").toString();
        
        String json = page.getRawText();
        try {
        	if(StringUtils.isNotBlank(json)){
        		json = json.substring(json.indexOf("(")+1,json.lastIndexOf(");"));
        		JSONObject obj = GsonUtils.fromJson(json, JSONObject.class);
        		String tg = obj.getJSONObject("data").getString("tg");
        		if(tg.contains("普通话")){
        			netTVModel.setLang("普通话");
        		}
            }
		} catch (Exception e) {
		}
        	
        extraMap.put(NetTVModel.class.getName(), netTVModel);
		//添加导演请求
        PageUtils.addTargetRequest(page, String.format(DIRECTOR_URL, tvId), extraMap);
	}

	/** basicinfoProcess(剧基本信息)   
	 * 创建人：张晓   
	 * 创建时间：2020年1月3日 下午12:04:26    
	 * 修改备注： 
	 * @param page  
	 */
	@SuppressWarnings("unchecked")
	private void basicinfoProcess(Page page,Map<String,Object> extraMap) {
		String json = page.getRawText();
		if (StringUtils.isEmpty(json))
            throw new SpiderException(SpiderExceptionCode.DataInvalideException);
		Map<String,Object> jsonMap = GsonUtils.fromJson(json, Map.class);
		Map<String,Object> dataMap = (Map<String,Object>)jsonMap.get("data");
		if (null == dataMap)
            throw new SpiderException(SpiderExceptionCode.DataInvalideException);
		List<Map<String,Object>> list = (List<Map<String,Object>>)dataMap.get("list");
		
		for(Map<String,Object> l : list){
			
			Map<String,Object> everyExtraMap = new HashMap<String,Object>();
			//网站年份
			String year = extraMap.get("year").toString();
			//网站地区
			String area = extraMap.get("area").toString();
			
			String albumid = l.get("albumId")==null?null:l.get("albumId")+StringUtils.EMPTY;
			String playUrl = l.get("playUrl")==null?null:l.get("playUrl")+StringUtils.EMPTY;
			String tvId = l.get("tvId")==null?null:l.get("tvId")+StringUtils.EMPTY;
			
			everyExtraMap.put("tvId", tvId);
			
			//主演
			String actor = l.get("secondInfo")==null?null:l.get("secondInfo")+StringUtils.EMPTY;
			actor = actor.replace("主演:", StringUtils.EMPTY).replace("/", ",").replace(" ", "");
			
			//剧名
			String tvname = l.get("name")==null?null:l.get("name")+StringUtils.EMPTY;
			//集数
			String sets = l.get("videoCount")==null?null:l.get("videoCount")+StringUtils.EMPTY;
			//类型
			String label = StringUtils.EMPTY;
			List<Map<String,Object>> labelList = (List<Map<String,Object>>)l.get("categories");
			for(Map<String,Object> labelMap : labelList){
				if(!labelMap.get("name").equals("网剧")){
					label += ","+labelMap.get("name");
				}
			}
			if(StringUtils.isNotBlank(label)){
				label = label.substring(1);
			}
			//上映时间
			String showdate = l.get("issueTime") == null?null:
				DateUtils.transferLongToDateString(DateUtils.DAY_FORMAT, 
						Long.parseLong(new BigDecimal(l.get("issueTime").toString()).toPlainString()));
			//描述
			String description = l.get("description")==null?null:l.get("description")+StringUtils.EMPTY;
			
			//评分
			String score = l.get("score")==null?null:l.get("score")+StringUtils.EMPTY;
			
			NetTVModel netTVModel = new NetTVModel(Constants.TargetType
	                .OTHER_TYPE_ID, tvname, Constants.SiteName.IQIYI, sets, label, null,
	                area, showdate, null, actor, description, null, score, year);
			everyExtraMap.put(NetTVModel.class.getName(), netTVModel);
			PageUtils.addTargetRequest(page, playUrl, everyExtraMap);
			PeoplePageProcessor.addPeopleLink(page, albumid);
		}
	}

	/**
     * 详情页面解析
     *
     * @param page
     */
    private void detailProcess(Page page,Map<String,Object> extraMap)
    {
    	
        //网站tvid
        String tvId = extraMap.get("tvId").toString();
        
        extraMap.put("tvId", tvId);
        
        //获取地区，语言
        PageUtils.addTargetRequest(page, String.format(AEREA_URL, tvId), extraMap);
    }
	
	/**
     * 受众数据处理
     *
     * @param page
     * @return
     */
    private void audienceProcess(Page page)
    {
        PeoplePageProcessor.peopleProcess(page, Constants.TargetType.OTHER_TYPE_ID);
    }

    /**
     * 分城数据处理
     *
     * @param page
     * @return
     */
    private void cityProcess(Page page)
    {
        Map<String, Object> map = PeoplePageProcessor.getDataMap(page);
        List<Map<String, Object>> dataList = PeoplePageProcessor.getCityData(map);
        List<TVCityModel> tvList = new ArrayList<>(dataList.size());
        dataList.forEach(tvMap -> tvList.add(new TVCityModel(Constants.TargetType.OTHER_TYPE_ID,
                null, PeoplePageProcessor.getDataIdFromJson(map), tvMap.get("name").toString(),
                tvMap.get("value").toString())));
        page.putField(TVCityModel.class.getName(), tvList);
    }

    

    /**
     * 爬虫的初始地址
     *
     * @return
     */
    @Override
    protected String[] getStartRequestUrls()
    {
        return new String[]{START_URL};
    }

    /**
     * 爬取任务的标识
     *
     * @return
     */
    @Override
    protected String getSpiderType()
    {
        return "IQY_NETTV";
    }

}
